# 云筑集采的中标公告详情页数据消费者
import random
import time
import pymysql
import redis
import requests
from msg_queue import connect_message_queue
from user_agent_kuaidaili_proxy import get_random_ua
import datetime
import threading

redis_test_url = "redis://192.168.1.9:6379/5"
mysql45_config = {
            'host': "192.168.1.45",
            'user': "xzl",
            'password': "Xzl4kyk!",
            'database': "qcc_company_graph",
            'port': 3306,
            'charset': 'utf8mb4'
        }
proxy = {'http': 'http://t10635913038651:09ht71vf@tps152.kdlapi.com:15818/', 'https': 'http://t10635913038651:09ht71vf@tps152.kdlapi.com:15818/'}
REDIS_PASSWORD = None


class YzwConsume(object):
    def __init__(self):
        self.company_source = 'www.yzw.cn'
        self.redis_url = redis_test_url
        self.mysql_conn = pymysql.connect(**mysql45_config)
        self.mysql_cursor = self.mysql_conn.cursor()
        self.redis_conn = redis.Redis(host='localhost', port=6379, db=5, password=REDIS_PASSWORD)

    def distinct_id_redis(self, company_id):
        if self.redis_conn.sismember('yzw_company_set', company_id):
            print('company_id已存在跳过')
            return True
        else:
            self.redis_conn.sadd('yzw_company_set', company_id)
            return False

    def distinct_retry_data_redis(self, url):
        if self.redis_conn.sismember('retry_yzw_tender_set', url):
            print('retry_url已存在跳过')
            return True
        else:
            self.redis_conn.sadd('retry_yzw_tender_set', url)
            return False

    def save_company_mysql(self,company_name, company_id):
        # 查询名称是否已存在
        select_sql = "SELECT * FROM company_info WHERE company_id='%s'" % company_id
        self.mysql_cursor.execute(select_sql)
        result = self.mysql_cursor.fetchone()

        if result:
            # 找到重复数据,跳过插入
            print("%s公司已存在" % company_name)
        else:
            insert_sql = "INSERT INTO company_info (source, company_name, company_id,  create_on) " \
                         "VALUES (%s, %s, %s, NOW())"
            self.mysql_cursor.execute(insert_sql, (self.company_source,company_name, company_id))
            self.mysql_conn.commit()


    def save_tender_mysql(self,tender_project, tender_id, winner_id, complete_time ,project_area):
        insert_sql = "INSERT INTO tender_info (source, tender_project, tender_id, winner_id, complete_time ,project_area, create_on) " \
                     "VALUES (%s, %s, %s,%s, %s, %s, NOW())"
        self.mysql_cursor.execute(insert_sql, (self.company_source,tender_project, tender_id, winner_id, complete_time ,project_area))
        self.mysql_conn.commit()


    def request_get_method(self, source, tender_code):
        url = 'https://yzmtg.yzw.cn/portal/tender/winner/detail?&source={}&tenderCode={}'.format(source, tender_code)
        # 获取列表页图片数据
        new_user_agent = get_random_ua()
        headers = {
            'Accept': 'application/json, text/plain, */*',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Boxid': 'Bch0HrOBWJtqaemUI532aBOkENVj6EpAj2xwJU2AHPt62E7+kioqRNdHxWsxrNhbQQdZhP6vHtKK6hm3chPnPDw==',
            'Cookie': 'yzw-auac-token=7T37OMnWmNbc9GWB9yrP3Z01fetxSzVLIO%2Bcwm54Hd0LMxUaV2NAMhfqaxbpEMNqlFALYM3Db%2FVUaiJ0sjhlqg%3D%3D; LoginRequestKey=3E3D740621312E802EFA2539E916237D66E35554B9378271E0793311458E409A286D5E7C7FDDA90BA3AFDBE59925BB91FE4B85BEE9E2C7F85D400DD8A87F7381A891B14446194F08B107FBCB052808580AFFCD372434D278BC5D3C0EF9CDA9FD8814B21E6ACD8F51A306433F885D2CEA45BE1C5FA934A30D86357727A8C27C32; web.auth.yzw=D2773E432B9AFF6EC8B7B81DA01710DD21B03095F325EE3ABD3B535C69906CE725085D7208A8F76FFACD9BD3D3079E9A8310CDCCB1C19CF1031D5B8CD95056A8939AFD057416591CCA52DECA175E67F59A587EBC8857B332955E3A17F5D496D1B20CB13678F8A552FF601B2A96392FCC56943902B731A532833326619FC5754784907DD5381CE5A81263E63D16E42BF52BA2829D2AC317E516ABA6F22683A30010CF0FF8FF0F83367EA114F58A101E2AEBA051528D248B8E91F6AF7FD4683CF0C1FF4807A3F3AA35D38701546745CB996321D086AA0CE1B5283803249A396628B42892A223A7561A0141341BC6CD042D63B2079811BF2DC5519FB113498E8CF889AA2B300A92324CA8024BAAD18B73D0866C6AF46E9A3341922631876BE524BF23003EEC50C1E399788BB430D15842885C4813FAF461806D931798471E7CD70FA7AE6114A3290D6EFB7DFF9EF4259F76484B36D0E271F146B64065DECCBC4607AF5D29AA6F3B21ADA171E59EC1E8DC9BCD968C65BAFCE0C427506308DA27C8FA887B1FEF7B371DBEADFB1D751BE19897A00F94C43849264974B494A0637369C09E2E0F9DC50C0238969F02FD9A725931A225340019C37E87A51F20060319A0224C521B8739091BBC8B9F89F34948CB6504C15C128F32B8A363C086578817A6B5B6F7F3FE52E600B334A73584C21134F1B3CADE83A3140C41C0D5FF4BAF19B53E0708ED2EBD198FD61124845F7A55BAA8592F6F88942CE90E462293401F24CBFA1BAFAB878F5FB53C4203DC569FD7FBED17A36715F4298B3496BE2BB5EBFC66721DC82FBD592965F8E00AEF9C7997D549C0D8C0AF98129FA43A10A0C47D50A12F83492FB7C4CA4A3833691EAA171B907F55FFEA1528C6F4DC30E95399ED2234828E5D398E56AD17E33B0004DE9969EB5D6C41696FF69DD0521B4466CB85F23ECE52BB26599C95E54DEC8C48305BB8654EF0EA79C1A31D22F101C4CA06F7E7CDBFFE8159C2BD979AE3B8A4FA76ADDB58781AF98B75363AE59F3703E595658D2E4E3AEAF5FFA3CA6D2F3F056D9B24CC; HWWAFSESTIME=1706067575419; HWWAFSESID=33a877f6b84acb1065',
            'Origin': 'https://xy.yzw.cn',
            'Referer': 'https://xy.yzw.cn/sj/win-bid-detail?source=1&tenderCode=cscec23102301103',
            'Sec-Fetch-Dest': 'empty',
            'Sec-Fetch-Mode': 'cors',
            'Sec-Fetch-Site': 'same-site',
            'User-Agent': new_user_agent,
            'X-Auth': 'HwWCX501fAXIV8885hlyjVnpnNu2Zii3iqYMReXpMkVTTgRObf8cF8TOjZqw3aM83QV6GlnEjzeUL7fsWHZGtn67SNqKaQZrun5B6I0wboQJEzZOXWY7cnWK+QGN4iIAjxHp84RLEo0U2T6KHfHeCxoDPf5uEubTouBBtL9KAPY=',
            'X-Yzw-Auth-Token': '7T37OMnWmNbc9GWB9yrP3Z01fetxSzVLIO%2Bcwm54Hd0LMxUaV2NAMhfqaxbpEMNqlFALYM3Db%2FVUaiJ0sjhlqg%3D%3D'
        }
        response = requests.get(url, headers=headers, proxies=proxy)
        # response = requests.post(URL, headers=headers, data=data)
        print('response', response.status_code)
        response.encoding = 'utf-8'
        json_data = response.json()
        print('response.url', response.url)
        print('json_data', json_data)
        return json_data


    def parse_data(self,data):
        res = data.get('data', None)
        if not res:
            return False
        project_area = [item.get('name') for item in res.get('areaList') if item.get('name') is not None]
        project_area = ','.join(project_area)
        complete_time = res.get('completeTime', None)
        tender_project = res.get('name', None)
        tender_name = res.get('tenderCompanyName', None)
        tender_id = res.get('tenderCompanyId', None)
        winner_ids =[item.get('companyId') for item in res.get('tenderWinnerSupplies') if item.get('companyId') is not None]
        winner_names = [item.get('companyName') for item in res.get('tenderWinnerSupplies') if item.get('companyName') is not None]
        print('project_area', project_area,'complete_time', complete_time,'tender_project', tender_project,'winner_ids', winner_ids,'winner_names', winner_names,)
        # redsi去重
        result = self.distinct_id_redis(tender_id)
        if not result:
            #新的
            self.save_company_mysql(tender_name, tender_id)
        else:
            print('招标方已存在')

        for i, winner_id in enumerate(winner_ids):
            re = self.distinct_id_redis(winner_id)
            if not re:
                # 新的
                self.save_company_mysql(winner_names[i], winner_id)
                self.save_tender_mysql(tender_project, tender_id, winner_id, complete_time, project_area)
            else:
                self.save_tender_mysql(tender_project, tender_id, winner_id, complete_time, project_area)


    def consume_message(self):
        thread_id = threading.current_thread().ident
        p = connect_message_queue('yzw_tender_queue', url=self.redis_url, maxsize=10000, lazy_limit=True)
        while p.qsize() > 0:
            json_msg = p.get()
            print('json_msg', json_msg)
            if not json_msg:
                continue

            print("Thread {} start ".format(thread_id))
            source = json_msg.get('source')
            tender_code = json_msg.get('tender_code')
            try:
                response = self.request_get_method(source, tender_code)
            except Exception as e:
                retry_url_queue_name = 'yzw_retry_queue'
                retry_data = tender_code+ '_' + str(source)
                retry_result = self.distinct_retry_data_redis(retry_data)
                if retry_result:
                    continue
                q = connect_message_queue(retry_url_queue_name, url=self.redis_url, maxsize=10000, lazy_limit=True)
                q.put(data)
                continue

            self.parse_data(response)
            # time.sleep(1)
            print("Thread {} finish ".format(thread_id))

    def run(self):
        # 创建多个消费线程
        # self.consumers_threads = []
        self.consume_message()
        # for i in range(0):
        #     t = threading.Thread(target=self.consume_message)
        #     t.start()
        #     self.consumers_threads.append(t)

    def __del__(self):
        self.mysql_cursor.close()
        self.mysql_conn.close()



yzw = YzwConsume()
yzw.run()

# data={'code': 200, 'data': {'abolished': False, 'areaList': [{'childNode': [], 'code': '110000', 'name': '北京市'}, {'childNode': [], 'code': '1115', 'name': '全国'}, {'childNode': [], 'code': '120000', 'name': '天津市'}, {'childNode': [], 'code': '130000', 'name': '河北省'}, {'childNode': [], 'code': '140000', 'name': '山西省'}, {'childNode': [], 'code': '150000', 'name': '内蒙古自治区'}, {'childNode': [], 'code': '210000', 'name': '辽宁省'}, {'childNode': [], 'code': '220000', 'name': '吉林省'}, {'childNode': [], 'code': '230000', 'name': '黑龙江省'}, {'childNode': [], 'code': '310000', 'name': '上海市'}, {'childNode': [], 'code': '320000', 'name': '江苏省'}, {'childNode': [], 'code': '330000', 'name': '浙江省'}, {'childNode': [], 'code': '340000', 'name': '安徽省'}, {'childNode': [], 'code': '350000', 'name': '福建省'}, {'childNode': [], 'code': '360000', 'name': '江西省'}, {'childNode': [], 'code': '370000', 'name': '山东省'}, {'childNode': [], 'code': '410000', 'name': '河南省'}, {'childNode': [], 'code': '420000', 'name': '湖北省'}, {'childNode': [], 'code': '430000', 'name': '湖南省'}, {'childNode': [], 'code': '440000', 'name': '广东省'}, {'childNode': [], 'code': '450000', 'name': '广西壮族自治区'}, {'childNode': [], 'code': '460000', 'name': '海南省'}, {'childNode': [], 'code': '500000', 'name': '重庆市'}, {'childNode': [], 'code': '510000', 'name': '四川省'}, {'childNode': [], 'code': '520000', 'name': '贵州省'}, {'childNode': [], 'code': '530000', 'name': '云南省'}, {'childNode': [], 'code': '540000', 'name': '西藏自治区'}, {'childNode': [], 'code': '610000', 'name': '陕西省'}, {'childNode': [], 'code': '620000', 'name': '甘肃省'}, {'childNode': [], 'code': '630000', 'name': '青海省'}, {'childNode': [], 'code': '640000', 'name': '宁夏回族自治区'}, {'childNode': [], 'code': '650000', 'name': '新疆维吾尔自治区'}, {'childNode': [], 'code': '7759', 'name': '台湾省'}, {'childNode': [], 'code': '7785', 'name': '香港'}, {'childNode': [], 'code': '7806', 'name': '澳门'}], 'completeTime': '2024-01-24 08:37:32', 'endTime': '2023-11-25 10:30:00', 'isBid': False, 'isSignUp': False, 'managerUserId': '852135', 'managerUserName': '姜宏伟', 'method': 1, 'name': '81-装饰-2023年度-2025年度-全国区域地坪漆联采-招标采购', 'orgCode': '00010100005000020007', 'publishDate': '2023-11-20 08:49:45', 'qualificationType': 0, 'signUpEndDate': '2023-11-25 10:30:00', 'source': 1, 'status': 3, 'tenderCategoryList': [{'code': '00010016', 'name': '其他类'}], 'tenderCode': 'cscec23111700216', 'tenderCompanyId': '50036628', 'tenderCompanyName': '装饰分公司', 'tenderEndDate': '2023-12-18 09:00:00', 'tenderId': '3214377', 'tenderOrganizationId': '3523', 'tenderOrganizationName': '装饰分公司', 'tenderProjectList': [], 'tenderWinnerSupplies': [{'companyId': '1022658', 'companyName': '中建西部建设新材料科技有限公司'}, {'companyId': '1050150', 'companyName': '北京盛源新业科技有限公司'}, {'companyId': '1102280', 'companyName': '湖南伟人建设工程有限公司'}, {'companyId': '125611', 'companyName': '湖北工程建设总承包有限公司'}], 'type': 0}, 'success': True}
# yzw = YzwConsume()
# yzw.parse_data(data)













